The goals / steps of this project are the following:
import glob
import time
import cv2
import numpy as np
from sklearn.svm import LinearSVC
from skimage.feature import hog
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from scipy.ndimage.measurements import label
from moviepy.editor import VideoFileClip
from IPython.display import HTML
from collections import deque
import warnings
warnings.filterwarnings("ignore")
%matplotlib inline
test_images = glob.glob('test_images/*.jpg')
cars = glob.glob('vehicles/**/*.png')
notcars = glob.glob('non-vehicles/**/*.png')
#print (len(cars))
#print (len(notcars))
#print (len(test_images))
# Just for fun choose random car / not-car indices and plot example images
car_ind = np.random.randint(0, len(cars))
notcar_ind = np.random.randint(0, len(notcars))
# Read in car / not-car images
car_image = mpimg.imread(cars[car_ind])
notcar_image = mpimg.imread(notcars[notcar_ind])
# Plot the examples
fig = plt.figure()
plt.figure(figsize=(20,20))
plt.subplot(121)
plt.imshow(car_image)
plt.title('Example Car Image',fontsize=20)
plt.subplot(122)
plt.imshow(notcar_image)
plt.title('Example Not-car Image',fontsize=20)
plt.savefig('output_images/Examples.png')
def convert_color(img, conv='RGB2YCrCb'):
if conv == 'RGB2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
if conv == 'BGR2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
if conv == 'RGB2LUV':
return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
if conv == 'RGB2HSV':
return cv2.cvtColor(img,cv2.COLOR_RGB2HSV)
# Essentially a resize function
def bin_spatial(img, size=(32, 32)):
color1 = cv2.resize(img[:, :, 0], size).ravel()
color2 = cv2.resize(img[:, :, 1], size).ravel()
color3 = cv2.resize(img[:, :, 2], size).ravel()
return np.hstack((color1, color2, color3))
def color_hist(img, nbins=32): #bins_range=(0, 256)
# Compute the histogram of the color channels separately
channel1_hist = np.histogram(img[:,:,0], bins=nbins)
channel2_hist = np.histogram(img[:,:,1], bins=nbins)
channel3_hist = np.histogram(img[:,:,2], bins=nbins)
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
# Return the individual histograms, bin_centers and feature vector
return hist_features
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
# Make a copy of the image
imcopy = np.copy(img)
# Iterate through the bounding boxes
for bbox in bboxes:
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
# Return the image copy with boxes drawn
return imcopy
def get_hog_features(img, orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True):
# Call with two outputs if vis==True
if vis == True:
features, hog_image = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=False,
visualise=vis, feature_vector=feature_vec)
return features, hog_image
# Otherwise call with one output
else:
features = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=False,
visualise=vis, feature_vector=feature_vec)
return features
# Define a function to extract features from a list of images
# Have this function call bin_spatial() and color_hist()
def extract_features(imgs, color_space='RGB', spatial_size=(32, 32),
hist_bins=32, orient=9,
pix_per_cell=8, cell_per_block=2, hog_channel=0,
spatial_feat=True, hist_feat=True, hog_feat=True):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
file_features = []
# Read in each one by one
image = mpimg.imread(file)
# apply color conversion if other than 'RGB'
if color_space != 'RGB':
if color_space == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif color_space == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif color_space == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif color_space == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif color_space == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else: feature_image = np.copy(image)
if spatial_feat == True:
spatial_features = bin_spatial(feature_image, size=spatial_size)
file_features.append(spatial_features)
if hist_feat == True:
# Apply color_hist()
hist_features = color_hist(feature_image, nbins=hist_bins)
file_features.append(hist_features)
if hog_feat == True:
# Call get_hog_features() with vis=False, feature_vec=True
if hog_channel == 'ALL':
hog_features = []
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
else:
hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
pix_per_cell, cell_per_block, vis=False, feature_vec=True)
# Append the new feature vector to the features list
file_features.append(hog_features)
features.append(np.concatenate(file_features))
# Return list of feature vectors
return features
def find_cars(img, ystart, ystop, scale,
svc, X_scaler, color_space,
orient, pix_per_cell,
cell_per_block, hog_channel,
spatial_size,
hist_bins, xlimit,all_rectangles=False):
'''Extracts features from image and trains SVM model on those features.
'''
draw_img = np.copy(img)
img = img.astype(np.float32) / 255
rectangles = []
img_tosearch = img[ystart:ystop, :, :]
ctrans_tosearch = convert_color(img_tosearch, conv='RGB2'+color_space)
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1] / scale), np.int(imshape[0] / scale)))
ch1 = ctrans_tosearch[:, :, 0]
ch2 = ctrans_tosearch[:, :, 1]
ch3 = ctrans_tosearch[:, :, 2]
# Define blocks and steps as above
nxblocks = (ch1.shape[1] // pix_per_cell) +1
nyblocks = (ch1.shape[0] // pix_per_cell) -1
nfeat_per_block = orient * cell_per_block ** 2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell) - 1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
nysteps = (nyblocks - nblocks_per_window) // cells_per_step
# Compute individual channel HOG features for the entire image
if hog_channel == 'ALL':
hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
else:
hog1 = get_hog_features(hog_channel, orient, pix_per_cell, cell_per_block, feature_vec=False)
for xb in range(xlimit,nxsteps):
for yb in range(nysteps):
ypos = yb * cells_per_step
xpos = xb * cells_per_step
hog_feat1 = hog1[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
if hog_channel == 'ALL':
hog_feat2 = hog2[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
hog_feat3 = hog3[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
else:
hog_features = hog_feat1
xleft = xpos * pix_per_cell
ytop = ypos * pix_per_cell
subimg = cv2.resize(ctrans_tosearch[ytop:ytop + window, xleft:xleft + window], (64, 64))
spatial_features = bin_spatial(subimg, size=spatial_size)
hist_features = color_hist(subimg, nbins=hist_bins)
test_features = X_scaler.transform(
np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
test_prediction = svc.predict(test_features)
if test_prediction == 1 or all_rectangles:
xbox_left = np.int(xleft * scale)
ytop_draw = np.int(ytop * scale)
win_draw = np.int(window * scale)
# It is best to save regions for now
rectangles.append(((xbox_left, ytop_draw + ystart),
(xbox_left + win_draw, ytop_draw + win_draw + ystart)))
return rectangles
# Create random index point for each list of car and notcars filename
car_ind = np.random.randint(0, len(cars))
notcar_ind = np.random.randint(0, len(notcars))
# Read in image from cars,notcars list using random indices from above
car_img = mpimg.imread(cars[car_ind])
noncar_img = mpimg.imread(notcars[notcar_ind])
# Create hog for the random images selected
features,car_dst = get_hog_features(car_img[:,:,2], 9, 8, 8, vis=True, feature_vec=True)
features,noncar_dst = get_hog_features(noncar_img[:,:,2], 9, 8, 8, vis=True, feature_vec=True)
# Visualize
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20,20))
ax1.imshow(car_img)
ax1.set_title('Car Image', fontsize=20)
ax2.imshow(car_dst, cmap='gray')
ax2.set_title('Car HOG', fontsize=20)
ax3.imshow(noncar_img)
ax3.set_title('Non-Car Image', fontsize=20)
ax4.imshow(noncar_dst, cmap='gray')
ax4.set_title('Non-Car HOG', fontsize=20)
# Define the settings used for the feature extraction
color_space='YCrCb'
orient=8
pix_per_cell=8
cell_per_block=2
hog_channel='ALL'
spatial_size=(32, 32)
hist_bins=32
spatial_feat=True
hist_feat=True
hog_feat=True
t=time.time()
## Extract the features for all the car and not car images
car_features = extract_features(cars, color_space=color_space, spatial_size=spatial_size,
hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)
notcar_features = extract_features(notcars, color_space=color_space, spatial_size=spatial_size,
hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)
t2 = time.time()
print(round(t2-t, 5), 'Seconds to extract features')
# Create an array stack of feature vectors
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = StandardScaler().fit(X).transform(X)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
# Use 20% test size
rand_state=np.random.randint(0,100)
X_train, X_test, y_train, y_test = train_test_split(scaled_X, y, test_size=0.2, random_state=rand_state)
print('Using spatial binning of:',spatial_size,
'and', hist_bins,'histogram bins')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
The code block illustrates the overlapping regions that is being used to search for a car
img = mpimg.imread('test_images/test6.jpg')
rects = []
y_start= [400,425,400,450,500,425]
y_stop = [500,550,600,650,700,725]
scale = [1.0,1.25,1.5,2.0,2.5,3.0]
xlimit = [40,30,25,17,15,10]
num=1
fig= plt.figure(figsize=(20,20))
# Iterate in parallel y_start,y_stop_,scale. Draw each square regardless of detection
for ystart,ystop,scale,xlim in zip(y_start,y_stop,scale,xlimit):
rect=find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim,all_rectangles=True)
plot=draw_boxes(img,rect,thick=2)
ax=fig.add_subplot(3,2,num,autoscale_on=True)
ax.imshow(plot)
ax.set_title('Region '+str(num),fontsize=40)
rects.append(rect)
num=num+1
plt.tight_layout()
plt.show()
Note that not all regions overlap the cars in this specific test image, which is why Regions 4-6 have no boundary boxes in the image.
img = mpimg.imread('test_images/test6.jpg')
rects = []
# same settings as previous block, redundant, but used to clarify what is going on
y_start= [400,425,400,450,500,425]
y_stop = [500,550,600,650,700,725]
scale = [1.0,1.25,1.5,2.0,2.5,3.0]
xlimit = [40,30,25,17,15,10]
# Iterate in parallel y_start,y_stop_,scale. This time only drawing the rectangles that detect a car
num=1
fig= plt.figure(figsize=(20,20))
for ystart,ystop,scale,xlim in zip(y_start,y_stop,scale,xlimit):
rect=find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim,all_rectangles=False)
plot=draw_boxes(img,rect,color=(0,255,0),thick=2)
ax=fig.add_subplot(3,2,num,autoscale_on=True)
ax.imshow(plot)
# If there are no boxes in rect then there were no cars detected in the region
if len(rect)==0:
detected='False'
else:
detected='True'
ax.set_title('Region '+str(num)+' Car detected= '+detected,fontsize=40)
rects.append(rect)
num=num+1
plt.tight_layout()
plt.show()
For this test image, region 3 seems to detect the cars in the image well.
y_start = 400
y_stop = 600
scale = 1.5
xlimit =25
car_windows =find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim,all_rectangles=False)
test_img_rects = draw_boxes(img, car_windows,color=(255,0,0),thick=2)
plt.figure(figsize=(20,20))
plt.imshow(test_img_rects)
plt.title('Squares Detecting Car',fontsize=40)
def add_heat(heatmap, bbox_list):
# Iterate through list of bboxes
for box in bbox_list:
# Add += 1 for all pixels inside each bbox
# Assuming each "box" takes the form ((x1, y1), (x2, y2))
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return heatmap
rects = []
y_start= [400,425,400,450,500,425]
y_stop = [500,550,600,650,700,725]
scale= [1.0,1.25,1.5,2.0,2.5,3.0]
xlimit = [40,30,25,17,15,10]
# Iterate in parallel y_start,y_stop_,scale. This time only passing the rectangle valus to the add_heat function
for ystart,ystop,scale,xlim in zip(y_start,y_stop,scale,xlimit):
rects.append(find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim))
rectangles = [item for sublist in rects for item in sublist]
heatmap_img = np.zeros_like(img[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
plt.figure(figsize=(25,25))
plt.imshow(heatmap_img, cmap='hot')
plt.title('Car Positions',fontsize=40)
def apply_threshold(heatmap, threshold):
# Zero out pixels below the threshold
heatmap[heatmap <= threshold] = 0
# Return thresholded map
return heatmap
#Apply threshold and to filter false positives
#np.set_printoptions(suppress=True)
threshed_heatmap_img = apply_threshold(heatmap_img, 2)
plt.figure(figsize=(25,25))
plt.imshow(threshed_heatmap_img, cmap='hot')
plt.title('Threshed Heat Map',fontsize=40)
labels = label(threshed_heatmap_img)
plt.figure(figsize=(25,25))
plt.imshow(labels[0], cmap='gray')
plt.title('Identify and label cars',fontsize=40)
print(labels[1], 'cars found')
def draw_labeled_bboxes(img, labels):
# Iterate through all detected cars
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
# Draw the box on the image
cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
# Return the image and final rectangles
return img, rects
# Draw bounding boxes on a copy of the image
draw_img, rect = draw_labeled_bboxes(np.copy(img), labels)
# Display the image
fig=plt.figure()
plt.figure(figsize=(32,24))
plt.subplot(1,2,1)
plt.imshow(draw_img)
plt.title('Car Detected',fontsize=40)
plt.subplot(1,2,2)
plt.imshow(heatmap_img,cmap='hot')
plt.title('Heat Map',fontsize=40)
plt.savefig('output_images/plotcompare.png')
#This pipeline is for single images.
def vehicle_detection_pipeline(img):
rects = []
y_start = [400,425,400,450,500,425]
y_stop = [500,550,600,650,700,725]
scale = [1.0,1.25,1.5,2.0,2.5,3.0]
xlimit = [40,30,25,17,15,10]
# Iterate in parallel y_start,y_stop_,scale. This time only passing the rectangle valus to the add_heat function
for ystart,ystop,scale,xlim in zip(y_start,y_stop,scale,xlimit):
rects.append(find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim))
rectangles = [item for sublist in rects for item in sublist]
heatmap_img = np.zeros_like(img[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
heatmap_img = apply_threshold(heatmap_img, 3)
labels = label(heatmap_img)
draw_img, rect = draw_labeled_bboxes(np.copy(img), labels)
return draw_img
fig = plt.figure(figsize=(20,16))
f.tight_layout()
for i in range(len(test_images)):
_img = mpimg.imread(test_images[i])
test_img=vehicle_detection_pipeline(_img)
ax=fig.add_subplot(3,2,i+1)
ax.imshow(test_img)
ax.set_title('Test Image '+str(i+1),fontsize=20)
plt.show()
#Similar to the single image pipeline but this incorporates memory system that remembers the last 8 frames
prev_frames=deque(maxlen=8)
def vehicle_detection_pipeline_video(img):
rects = []
y_start = [410,400,425,400,450,500,425]
y_stop = [460,500,550,600,650,700,725]
scale = [1.15,1.0,1.25,1.5,2.0,2.5,3.0]
xlimit = [40,30,25,17,15,10]
start=time.time()
for ystart,ystop,scale,xlim in zip(y_start,y_stop,scale,xlimit):
rects.append(find_cars(img, ystart, ystop, scale, svc,
X_scaler, color_space, orient,
pix_per_cell, cell_per_block, hog_channel,
spatial_size, hist_bins,xlim))
rectangles = [item for sublist in rects for item in sublist]
heatmap_img = np.zeros_like(img[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
heatmap_img = apply_threshold(heatmap_img, 3) # threshold op1
prev_frames.append(heatmap_img)#add current heatmap to memory
heatmap_img=sum(prev_frames)# calc new heatmap_img that uses last 8 frames
heatmap_img = apply_threshold(heatmap_img,2) # threshold op2
labels = label(heatmap_img)
draw_img, rect = draw_labeled_bboxes(np.copy(img), labels)
end=time.time()
print('Seconds between frames ',round(end-start,5))
return draw_img
output = 'Proj_5.mp4'
clip_test = VideoFileClip('project_video.mp4')#.subclip(10,30)
clip_test_out = clip_test.fl_image(vehicle_detection_pipeline_video)
%time clip_test_out.write_videofile(output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(output))